{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'  # default is 'last_expr'\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from random import shuffle, sample\n",
    "from collections import defaultdict\n",
    "\n",
    "import sys\n",
    "sys.path.append('../../../')  # CameraTraps repository base dir\n",
    "\n",
    "from data_management.megadb.megadb_utils import MegadbUtils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits_dir = '/Users/siyuyang/Library/CloudStorage/OneDrive-Microsoft/Projects/CameraTrap/Databases/megadb_splits_table'\n",
    "\n",
    "batch_12_dir = '/Users/siyuyang/Library/CloudStorage/OneDrive-Microsoft/Projects/CameraTrap/Databases/megadb_batch_12/after_anno'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "megadb_utils = MegadbUtils()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits_table = megadb_utils.get_splits_table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['alka_squirrels',\n",
       " 'amapa_1819',\n",
       " 'au_nt_gov_kerr',\n",
       " 'auckland_doc_2019',\n",
       " 'awc_190430',\n",
       " 'awc_202103',\n",
       " 'caltech',\n",
       " 'channel_islands_tnc',\n",
       " 'channel_islands_tnc_private',\n",
       " 'fws_butler_redrock',\n",
       " 'fws_hawaii_kauai_forest_birds_a24s',\n",
       " 'fws_lehnen_mice',\n",
       " 'idfg',\n",
       " 'idfg_swwlf_2020',\n",
       " 'islandconservation_midway_2020',\n",
       " 'islandconservation_rodents_2020',\n",
       " 'kays_emammal',\n",
       " 'mcshea_emammal',\n",
       " 'nacti',\n",
       " 'peaceparks_201908_humans',\n",
       " 'rspb_atkinson',\n",
       " 'rspb_gola',\n",
       " 'rspb_gola_2020',\n",
       " 'saola',\n",
       " 'saola_private',\n",
       " 'snapshot_camdeboo',\n",
       " 'snapshot_enonkishu',\n",
       " 'snapshot_karoo',\n",
       " 'snapshot_kgalagadi',\n",
       " 'snapshot_kruger',\n",
       " 'snapshot_mountain_zebra',\n",
       " 'snapshot_safari_private',\n",
       " 'snapshotserengeti',\n",
       " 'snapshotserengeti_private',\n",
       " 'ubc_fennell',\n",
       " 'umn_gomez',\n",
       " 'uw_gardner',\n",
       " 'wcs',\n",
       " 'wcs_private',\n",
       " 'wellington_nz',\n",
       " 'wpz_emammal_2018',\n",
       " 'zsl_borneo']"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(splits_table)\n",
    "sorted(splits_table.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_table = megadb_utils.get_datasets_table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(datasets_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_datasets = set(datasets_table.keys())\n",
    "datasets_w_splits = set(splits_table.keys())\n",
    "\n",
    "need_splitting = all_datasets - datasets_w_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bellevue_190602',\n",
       " 'bnf_20190624and0815',\n",
       " 'ena24',\n",
       " 'fws_hawaii_kauai_forest_birds_training',\n",
       " 'idfg_swwlf_2019',\n",
       " 'islandconservation_190705',\n",
       " 'islandconservation_200529',\n",
       " 'islandconservation_200529_private',\n",
       " 'nacti_private',\n",
       " 'parkscanada_20190715',\n",
       " 'parkscanada_garrow_201920_trains',\n",
       " 'rspb_gola_1819',\n",
       " 'sulross_2018',\n",
       " 'sulross_2019_spring',\n",
       " 'sulross_kitfox',\n",
       " 'trailguard_night_mara_190515',\n",
       " 'wii20190517',\n",
       " 'wiitigers',\n",
       " 'wps_190624',\n",
       " 'wps_210101',\n",
       " 'wpz_wolverine_labeled',\n",
       " 'wpz_wolverine_unlabeled'}"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "need_splitting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_boxes_at_locations(sequences):\n",
    "    counter = defaultdict(int)\n",
    "    for seq in sequences:\n",
    "        location = seq['location'] if 'location' in seq else 'NO_LOCATION'\n",
    "        num_boxes = 0\n",
    "        for im in seq['images']:\n",
    "            if 'bbox' in im:\n",
    "                num_boxes += len(im['bbox'])\n",
    "        counter[location] += num_boxes\n",
    "    return counter\n",
    "\n",
    "def get_proportions(counter, train, val, test):\n",
    "    # proportion of each split in terms of number of bounding boxes\n",
    "    assert sum((len(train), len(val), len(test))) == len(counter)\n",
    "    \n",
    "    total = sum(counter.values())\n",
    "    train_percent = 100 * sum([counter[location] for location in train]) / total\n",
    "    val_percent = 100 * sum([counter[location] for location in val]) / total\n",
    "    test_percent = 100 * sum([counter[location] for location in test]) / total\n",
    "    \n",
    "    print(f'Total {total} boxes: {train_percent:.2f}% train, {val_percent:.2f}% val, {test_percent:.2f}% test')\n",
    "\n",
    "def write_splits(dataset_name, train, val, test):\n",
    "    splits_obj = {\n",
    "        'dataset': dataset_name,\n",
    "        'train': sorted(train),\n",
    "        'val': sorted(val),\n",
    "        'test': sorted(test)\n",
    "    }\n",
    "\n",
    "    with open(os.path.join(splits_dir, f'splits_{dataset_name}.json'), 'w', encoding='utf8') as f:\n",
    "        json.dump(splits_obj, f, indent=1, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## alka_squirrels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(batch_12_dir, 'alka_squirrels.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "location_boxes\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "val = ['tagged_0', '2020_05_08_42C']\n",
    "test = ['2020_05_07_30C']\n",
    "train = [loc for loc in location_boxes.keys() if loc not in val and loc not in test and loc != 'NO_LOCATION']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 6394 boxes: 89.10% train, 5.15% val, 5.76% test\n"
     ]
    }
   ],
   "source": [
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits('alka_squirrels', train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## amapa_1819"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'amapa_1819'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {'C2012P031': 292,\n",
       "             'C2012P032': 629,\n",
       "             'C2012P041_A': 139,\n",
       "             'C2012P041_B': 27,\n",
       "             'C2012P06': 147,\n",
       "             'C2012P07': 59,\n",
       "             'C2012P081': 277,\n",
       "             'C2012P091': 153,\n",
       "             'C2012P12': 201,\n",
       "             'C2012P13': 124,\n",
       "             'C2012P16': 498,\n",
       "             'C2012P181': 627,\n",
       "             'C2012P242': 151,\n",
       "             'C2012P28': 529,\n",
       "             'C2017P2': 87,\n",
       "             'C2017P3': 482,\n",
       "             'C2017P4': 204,\n",
       "             'C2017P5': 155,\n",
       "             'C2017P6': 22,\n",
       "             'C2017P7_escondida': 88,\n",
       "             'CBF27': 321,\n",
       "             '2012P03': 262,\n",
       "             '2012P03_c2': 456,\n",
       "             '2012P04': 64,\n",
       "             '2012P04B': 99,\n",
       "             '2012P06': 97,\n",
       "             '2012P07': 95,\n",
       "             '2012P08': 330,\n",
       "             '2012P09': 84,\n",
       "             '2012P12': 116,\n",
       "             '2012P13': 416,\n",
       "             '2012P16': 148,\n",
       "             '2012P18': 1195,\n",
       "             '2012P24': 43,\n",
       "             '2012P27': 156,\n",
       "             '2012P28': 168,\n",
       "             '2017P2': 48,\n",
       "             '2017P7': 345,\n",
       "             'BF18': 45,\n",
       "             'BF27': 171,\n",
       "             'BF52': 157,\n",
       "             'BF74': 137})"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "location_boxes\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "val = ['2012P06', 'C2017P3', 'BF18']\n",
    "test = ['C2017P5', '2012P12']\n",
    "train = [loc for loc in location_boxes.keys() if loc not in val and loc not in test and loc != 'NO_LOCATION']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 9844 boxes: 90.91% train, 6.34% val, 2.75% test\n"
     ]
    }
   ],
   "source": [
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## au_nt_gov_kerr\n",
    "Incomplete location info - many with location 'unknown'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'au_nt_gov_kerr'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "location_boxes\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 19758 boxes: 95.14% train, 3.39% val, 1.47% test\n"
     ]
    }
   ],
   "source": [
    "val = ['AR', 'BN', 'WAD', 'WN']\n",
    "test = ['BP', 'MAN']\n",
    "train = [loc for loc in location_boxes.keys() if loc not in val and loc not in test and loc != 'NO_LOCATION']\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## auckland_doc_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'auckland_doc_2019'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "337"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'vBqd'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'vkR4'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "locations[0]\n",
    "shuffle(locations)\n",
    "locations[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 35529 boxes: 96.14% train, 2.38% val, 1.49% test\n"
     ]
    }
   ],
   "source": [
    "val = locations[:15]\n",
    "test = locations[15:30]\n",
    "train = locations[30:]\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## awc_202103"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'awc_202103'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "157"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 19467 boxes: 91.41% train, 3.79% val, 4.80% test\n"
     ]
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "shuffle(locations)\n",
    "\n",
    "val = locations[:8]\n",
    "test = locations[8:16]\n",
    "train = locations[16:]\n",
    "\n",
    "assert 'unknown' in train\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## channel_islands_tnc and channel_islands_tnc_private\n",
    "\n",
    "Since they come from a common set of locations, they should share one way of splitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'channel_islands_tnc'\n",
    "\n",
    "with open('/Users/siyuyang/Library/CloudStorage/OneDrive-Microsoft/Projects/CameraTrap/Databases/megadb_mdv5/channel_islands_tnc.json') as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name1 = 'channel_islands_tnc_private'\n",
    "\n",
    "with open('/Users/siyuyang/Library/CloudStorage/OneDrive-Microsoft/Projects/CameraTrap/Databases/megadb_mdv5/channel_islands_tnc_private.json') as f:\n",
    "    sequences1 = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes1 = count_boxes_at_locations(sequences1)\n",
    "len(location_boxes1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 143346 boxes: 70.45% train, 13.30% val, 16.25% test\n"
     ]
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "shuffle(locations)\n",
    "\n",
    "val = locations[:10] # more generous on the val/test proportion as we need more person/vehicle images in val/test overall\n",
    "test = locations[10:20]\n",
    "train = locations[20:]\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[s for s in test if s in location_boxes1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = [s for s in train if s in location_boxes1]\n",
    "val = [s for s in val if s in location_boxes1]\n",
    "test = [s for s in test if s in location_boxes1]\n",
    "\n",
    "len(train) + len(val) + len(test)\n",
    "write_splits(dataset_name1, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ena24\n",
    "\n",
    "No location info. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fws_butler_redrock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'fws_butler_redrock'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {'Ash1': 382,\n",
       "             'Ash2': 27,\n",
       "             'Ash3': 14,\n",
       "             'Ashwind': 244,\n",
       "             'G0': 51,\n",
       "             'G10': 18,\n",
       "             'G11': 8,\n",
       "             'G11c': 502,\n",
       "             'G2': 30,\n",
       "             'G3': 65,\n",
       "             'G4': 5,\n",
       "             'G5': 64,\n",
       "             'G6': 17,\n",
       "             'G7box': 487,\n",
       "             'G7c': 191,\n",
       "             'G8': 279,\n",
       "             'G9': 560,\n",
       "             'Jacks': 450,\n",
       "             'P1_2_3': 1048,\n",
       "             'P1_4d': 1612,\n",
       "             'P1_4f': 4240,\n",
       "             'P1_5': 2266,\n",
       "             'P3_4': 1469,\n",
       "             'Solar': 29863})"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 43892 boxes: 91.31% train, 5.34% val, 3.35% test\n"
     ]
    }
   ],
   "source": [
    "val = ['Ash3', 'G3', 'P1_5']\n",
    "test = ['P3_4']\n",
    "train = [loc for loc in location_boxes.keys() if loc not in val and loc not in test and loc != 'NO_LOCATION']\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fws_hawaii_kauai_forest_birds_a24s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'fws_hawaii_kauai_forest_birds_a24s'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 5738 boxes: 75.34% train, 14.69% val, 9.97% test\n"
     ]
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "shuffle(locations)\n",
    "\n",
    "val = locations[:15]\n",
    "test = locations[15:30]\n",
    "train = locations[30:]\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fws_lehnen_mice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'fws_lehnen_mice'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "80"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 7931 boxes: 89.17% train, 6.14% val, 4.69% test\n"
     ]
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "shuffle(locations)\n",
    "\n",
    "val = locations[:6]\n",
    "test = locations[6:10]\n",
    "train = locations[10:]\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## idfg_swwlf_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'idfg_swwlf_2020'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "517"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes) # few boxes/non-empty images at each location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 18458 boxes: 87.88% train, 5.83% val, 6.29% test\n"
     ]
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "shuffle(locations)\n",
    "\n",
    "val = locations[:25]\n",
    "test = locations[25:50]\n",
    "train = locations[50:]\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## islandconservation_190705, islandconservation_200529 and islandconservation_200529_private\n",
    "Do not have location info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## islandconservation_midway_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'islandconservation_midway_2020'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {'01W 8-5-17': 31,\n",
       "             '02W 8-5-17': 60,\n",
       "             '04W 8-5-17': 364,\n",
       "             '05W 8-6-17': 410,\n",
       "             '10E 8-7-17': 13,\n",
       "             '5W Aug 2017': 314,\n",
       "             'Bird 8-6-17': 33,\n",
       "             'Lehua bait station 5 8-23 to 11-15 2017': 1819,\n",
       "             'Lehua rat eats baby bird': 60,\n",
       "             'Lehua2016': 10816,\n",
       "             'Floreana_CAM1': 145,\n",
       "             'Floreana_CAM10': 83,\n",
       "             'Floreana_CAM11': 44,\n",
       "             'Floreana_CAM12': 408,\n",
       "             'Floreana_CAM13': 947,\n",
       "             'Floreana_CAM14': 5756,\n",
       "             'Floreana_CAM15': 1456,\n",
       "             'Floreana_CAM16': 165,\n",
       "             'Floreana_CAM2': 19739,\n",
       "             'Floreana_CAM3': 573,\n",
       "             'Floreana_CAM4': 93,\n",
       "             'Floreana_CAM5': 306,\n",
       "             'Floreana_CAM6': 51,\n",
       "             'Floreana_CAM7': 191,\n",
       "             'Floreana_CAM8': 91,\n",
       "             'Floreana_CAM9': 3,\n",
       "             'SantaCruz_CAM17': 361,\n",
       "             'SantaCruz_CAM18': 71,\n",
       "             'SantaCruz_CAM19': 743,\n",
       "             'SantaCruz_CAM31': 227,\n",
       "             'MidwayBoninNight_Sec07': 1237,\n",
       "             'MidwayBoninNight_Sec08': 2017,\n",
       "             'MidwayBoninNight_Sec11': 6656,\n",
       "             'MidwayBoninNight_Sec13': 886,\n",
       "             'MidwayBoninNight_Sec16': 10326,\n",
       "             'MidwayBoninNight_Sec18': 5392,\n",
       "             'MidwayBoninNight_Sec19': 1515,\n",
       "             'MidwayBoninNight_Sec20': 294,\n",
       "             'MidwayBoninNight_Sec21': 3347,\n",
       "             'MidwayBoninNight_Sec22': 104,\n",
       "             'MidwayBoninNight_Sec24': 1344,\n",
       "             'MidwayBoninNight_Sec33': 1518,\n",
       "             'MidwayBoninNight_Sec34': 614,\n",
       "             'MidwayBoninNight_Sec37': 1059})"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 81682 boxes: 95.11% train, 1.81% val, 3.08% test\n"
     ]
    }
   ],
   "source": [
    "val = ['04W 8-5-17', 'MidwayBoninNight_Sec13', 'SantaCruz_CAM31']\n",
    "test = ['Floreana_CAM15', 'MidwayBoninNight_Sec37']\n",
    "train = [loc for loc in location_boxes.keys() if loc not in val and loc not in test and loc != 'NO_LOCATION']\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## islandconservation_rodents_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'islandconservation_rodents_2020'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {'Aug2020/CAM003': 3,\n",
       "             'Aug2020/CAM007': 1,\n",
       "             'Aug2020/CAM010': 32,\n",
       "             'Aug2020/CAM012 BF': 6,\n",
       "             'Aug2020/CAM056': 39,\n",
       "             'Aug2020/CAM057': 13,\n",
       "             'Aug2020/CAM110': 41,\n",
       "             'Aug2020/cam052': 5,\n",
       "             'Aug2020/cam055': 3,\n",
       "             'July2020/CAM054': 3,\n",
       "             'July2020/CAM055': 3,\n",
       "             'July2020/CAM056': 116,\n",
       "             'July2020/CAM057': 6,\n",
       "             'July2020/CAM058': 19,\n",
       "             'Nov2020/TimelapseExport/CAM056': 12,\n",
       "             'Nov2020/TimelapseExport/CAM059': 3,\n",
       "             'Oct2020/cam001': 6,\n",
       "             'Oct2020/cam004': 9,\n",
       "             'Oct2020/cam012': 8,\n",
       "             'Oct2020/cam051': 3,\n",
       "             'Oct2020/cam052': 6,\n",
       "             'Oct2020/cam056': 18,\n",
       "             'Oct2020/cam057': 54,\n",
       "             'Oct2020/cam060': 289,\n",
       "             'Oct2020/cam101': 6,\n",
       "             'Oct2020/cam106': 0,\n",
       "             'Oct2020/cam108': 4,\n",
       "             'Oct2020/cam110': 3,\n",
       "             'Sep2020/cam05': 3,\n",
       "             'Sep2020/cam051': 6,\n",
       "             'Sep2020/cam052': 41,\n",
       "             'Sep2020/cam055': 6,\n",
       "             'Sep2020/cam056': 33,\n",
       "             'Sep2020/cam106': 9,\n",
       "             'Sep2020/cam109': 6,\n",
       "             'Sep2020/cam110': 5})"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 820 boxes: 83.90% train, 12.44% val, 3.66% test\n"
     ]
    }
   ],
   "source": [
    "val = ['Aug2020/CAM056', 'Oct2020/cam057', 'Sep2020/cam106']\n",
    "test = ['Oct2020/cam012', 'Nov2020/TimelapseExport/CAM059', 'July2020/CAM058']\n",
    "train = [loc for loc in location_boxes.keys() if loc not in val and loc not in test and loc != 'NO_LOCATION']\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rspb_atkinson\n",
    "Only 9 locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'rspb_atkinson'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {'Test': 3,\n",
       "             'OP211/ONWP335': 14,\n",
       "             'OP212/ONWP271': 52,\n",
       "             'OP206/ONWP362': 51,\n",
       "             'OP209/ONWP363': 186,\n",
       "             'OP210/ONWP272': 9,\n",
       "             'OP208/ONWP365': 3,\n",
       "             'OP207/ONWP334': 58,\n",
       "             'OP205/ONWP364': 83})"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 459 boxes: 76.03% train, 11.33% val, 12.64% test\n"
     ]
    }
   ],
   "source": [
    "val = ['OP212/ONWP271']\n",
    "test = ['OP207/ONWP334']\n",
    "train = [loc for loc in location_boxes.keys() if loc not in val and loc not in test and loc != 'NO_LOCATION']\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rspb_gola_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'rspb_gola_2020'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 15129 boxes: 88.18% train, 9.65% val, 2.17% test\n"
     ]
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "shuffle(locations)\n",
    "\n",
    "val = locations[:4]\n",
    "test = locations[4:8]\n",
    "train = locations[8:]\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## saola and saola_private"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'saola'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "935"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 100647 boxes: 85.93% train, 9.72% val, 4.35% test\n"
     ]
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "shuffle(locations)\n",
    "\n",
    "val = locations[:100]\n",
    "test = locations[100:150]\n",
    "train = locations[150:]\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "974"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name1 = 'saola_private'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name1}.json')) as f:\n",
    "    sequences1 = json.load(f)\n",
    "\n",
    "location_boxes1 = count_boxes_at_locations(sequences1)\n",
    "len(location_boxes1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1021"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(location_boxes1.values()) # we only bbox labeled a subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "825"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = [loc for loc in location_boxes1 if loc not in val and loc not in test]\n",
    "len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name1, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boxes_val = 0\n",
    "for loc in val:\n",
    "    boxes_val += location_boxes1[loc]\n",
    "boxes_val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## umn_gomez"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'umn_gomez'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 76679 boxes: 82.86% train, 12.11% val, 5.03% test\n"
     ]
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "shuffle(locations)\n",
    "\n",
    "val = locations[:4]\n",
    "test = locations[4:6]\n",
    "train = locations[6:]\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## uw_gardner\n",
    "\n",
    "Mostly animals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'uw_gardner'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 9176 boxes: 93.79% train, 3.98% val, 2.23% test\n"
     ]
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "shuffle(locations)\n",
    "\n",
    "val = locations[:3]\n",
    "test = locations[3:6]\n",
    "train = locations[6:]\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## wellington_nz\n",
    "LILA dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'wellington_nz'\n",
    "\n",
    "with open(os.path.join(batch_12_dir, f'{dataset_name}.json')) as f:\n",
    "    sequences = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "215"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_boxes = count_boxes_at_locations(sequences)\n",
    "len(location_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 215101 boxes: 74.53% train, 12.41% val, 13.06% test\n"
     ]
    }
   ],
   "source": [
    "locations = list(location_boxes.keys())\n",
    "shuffle(locations)\n",
    "\n",
    "val = locations[:25]\n",
    "test = locations[25:50]\n",
    "train = locations[50:]\n",
    "\n",
    "get_proportions(location_boxes, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_splits(dataset_name, train, val, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:cameratraps-batch-api]",
   "language": "python",
   "name": "conda-env-cameratraps-batch-api-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
